package fr.labri.harmony.analysis.jsep;

import java.io.File;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

public class DbToJson {

	public static void main(String[] args) throws Exception {
		
		int LIMIT_SIZE = 20000;
		
		Set<String> urls = new HashSet<String>();
		
		List<String> lines = Files.readAllLines(Paths.get("results-20131004-194837.csv"), Charset.forName("UTF-8"));
		lines.addAll(Files.readAllLines(Paths.get("results-20131004-194916.csv"), Charset.forName("UTF-8")));
		lines.addAll(Files.readAllLines(Paths.get("results-20131007-101712.csv"), Charset.forName("UTF-8")));
		lines.addAll(Files.readAllLines(Paths.get("results-20131007-101739.csv"), Charset.forName("UTF-8")));
		lines.addAll(Files.readAllLines(Paths.get("results-20131008-153041.csv"), Charset.forName("UTF-8")));
		lines.addAll(Files.readAllLines(Paths.get("results-20131008-153102.csv"), Charset.forName("UTF-8")));
		for(String line : lines) {
			String tk[] = line.split("\\,");
			urls.add(tk[0]);
		}
		List<String> shu = new ArrayList<String>(urls);
		Collections.shuffle(shu);
		System.out.println(shu.size());
		//Close connection
	
		ObjectMapper mapper = new ObjectMapper();
		ArrayNode sources = mapper.createArrayNode();

		for(String url : shu ) {
			ObjectNode source = mapper.createObjectNode();
			source.put("url", url);
			source.put("class", "JGitSourceExtractor");
			sources.add(source);
		}
		System.out.println(sources.size());
		//mapper.writeValue(new File("../corpus.bigquery"), sources);
		
	}


}
